# this file hold some utility methods

from google.appengine.api import urlfetch
from django.utils import simplejson
import urllib
import logging
import re
import sgmllib
import datetime
from StringIO import StringIO
import copy

from mmseg.algorithms import ComplexAlgorithm

# For Search Result Acquirement
#import myhttplib
import httplib
from xml.dom.minidom import getDOMImplementation

CLUSTER_SERVER = 'dcs.gceel.com'
CLUSTER_SERVER_3G = 'dcs.gceel.com'

def encode_multipart_formdata(fields):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'
    L = []
    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="%s"' % 'dcs.output.format')
    L.append('')
    L.append(fields['dcs.output.format'])

    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="%s"' % 'dcs.clusters.only')
    L.append('')
    L.append(fields['dcs.clusters.only'])
    
    #L.append('--' + BOUNDARY)
    #L.append('Content-Disposition: form-data; name="%s"' % 'dcs.algorithm')
    #L.append('')
    #L.append('haog-stc-en')#haog-fi-en, haog-stc-en, lingo-classic, rough-kmeans, stc-en    
    
    #logging.debug(fields['c2stream'])
    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="dcs.c2stream"; filename="web"')
    L.append('Content-Type: application/octet-stream')
    L.append('')
    L.append(fields['dcs.c2stream'])
    
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body

def encode_multipart_formdata_3g(fields):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'
    L = []
    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="%s"' % 'dcs.default.output')
    L.append('')
    L.append(fields['dcs.default.output'])

    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="%s"' % 'dcs.clusters.only')
    L.append('')
    L.append(fields['dcs.clusters.only'])

    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="%s"' % 'parameter-set-id')
    L.append('')
    L.append(fields['parameter-set-id'])
    
    #L.append('--' + BOUNDARY)
    #L.append('Content-Disposition: form-data; name="%s"' % 'dcs.algorithm')
    #L.append('')
    #L.append('lingo')#haog-fi-en, haog-stc-en, lingo-classic, rough-kmeans, stc-en    
    
    #logging.debug(fields['c2stream'])
    L.append('--' + BOUNDARY)
    L.append('Content-Disposition: form-data; name="c2stream"; filename="web"')
    L.append('Content-Type: application/octet-stream')
    L.append('')
    L.append(fields['c2stream'])
    
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body

def seal_query_xml(query, documents):
    logging.debug(query)
    impl = getDOMImplementation()
    doc = impl.createDocument(None, "searchresult", None)
    top_element = doc.documentElement

    query_element = doc.createElement("query")
    top_element.appendChild(query_element)
    text = doc.createTextNode(query)
    query_element.appendChild(text)

    id = 0
    for document in documents:
        document_element = doc.createElement("document")
        document_element.setAttribute('id', str(id))

        title_element = doc.createElement("title")
        document_element.appendChild(title_element)
        ttext = doc.createTextNode( document['Title'] )
        title_element.appendChild(ttext)

        snippet_element = doc.createElement("snippet")
        document_element.appendChild(snippet_element)
        stext = doc.createTextNode( mmseg(document['Summary']) )
        snippet_element.appendChild(stext)

        top_element.appendChild(document_element)
        id += 1

    seal = doc.toxml('utf-8')
    return seal

def get_cluster_results(c2stream):
    server = CLUSTER_SERVER #'http://128.36.117.51/rest/processor'
    #params = locals()
    params = {}
    params['dcs.c2stream'] = c2stream
    params['dcs.output.format'] = 'JSON'
    params['dcs.clusters.only'] = 'true'
    return _cluster_carrot(server, params)

def _cluster_carrot(server, form_fields):
    content_type, body = encode_multipart_formdata(form_fields)    

    h = httplib.HTTP(server, 8080)#'128.36.117.51'
    logging.debug(server)
    h.putrequest('POST', '/dcs/rest')
    h.putheader('content-type', content_type)
    h.putheader('content-length', str(len(body)))
    h.endheaders()
    h.send(body)
    errcode, errmsg, headers = h.getreply()
    #result = h.file.read()
    response = StringIO(h.file.read())
    #logging.debug(errmsg)
    result = simplejson.load(response)
    return result

def get_cluster_results_3g(c2stream):
    server = CLUSTER_SERVER_3G #'http://128.36.117.51/rest/processor'
    #params = locals()
    params = {}
    params['c2stream'] = c2stream
    params['dcs.default.output'] = 'json'
    params['dcs.clusters.only'] = 'true'
    params['parameter-set-id'] = 'ultra-fast'
    return _cluster_carrot_3g(server, params)

def _cluster_carrot_3g(server, form_fields):
    content_type, body = encode_multipart_formdata_3g(form_fields)    
    logging.debug(server)
    h = httplib.HTTP(server)#'128.36.117.51'
    h.putrequest('POST', '/rest/processor')
    h.putheader('content-type', content_type)
    h.putheader('content-length', str(len(body)))
    h.endheaders()
    h.send(body)
    errcode, errmsg, headers = h.getreply()
    #result = h.file.read()
    response = StringIO(h.file.read())
    result = simplejson.load(response)
    return result

'''
    Get N Google Search Results, N= 100, 200, 300, 500, 1000
'''
def get_N_google_search_results(q, N):
    if N <= 100:
        N = 100
    N = (int)(N/100)
    
    result = []
    for i in range(0, N):
        try:
            result += get_google_search_results(q, i*100)
            logging.debug('results: ' + str(len(result)) )
        except:
            continue
    return result

'''
    Get Google Search Results
'''
def get_google_search_results(q, start = 0, num = 100, complete = 1, hl = "en", sa ='N',
                              oe = 'UTF-8', ie = 'UTF-8'):
    #base_url = u'http://www.google.com/search?' #num=30&complete=1&hl=en&q=apple&start=30&sa=N
    params = locals()
    result = _query_google(params)
    return result

def _query_google(params):
    params['q'] = params['q'].encode('utf-8')
    #logging.debug(params['q'])
    payload = urllib.urlencode(params)
    path = "/search?" + payload
    #logging.debug(path)
    conn = httplib.HTTPConnection("www.google.com")
    #headers = {"User-agent": "Mozilla/5.0"}
    conn.request("GET", path, None, None)
    response = conn.getresponse()
    data = response.read()
    conn.close()
    result = data
    result = parser_google_web2(result)
    return result

def parser_google_web2(data):
    udata = data.decode('utf-8')
    results = []
    items = udata.split('<li class')
    for item in items:
        start = item.find('<a href="') + len('<a href="')
        item = item[start:]
        end = item.find('" class=l>')
        end2 = item.find(">")
        if end < 0 or end2 != end + 9:
            continue
        ClickUrl = item[:end]
        
        item = item[end+1:]
        start = item.find('>') + 1
        end = item.find('</a>')
        if end < 0:        
            continue
        Title = item[start:end]

        item = item[end+4:]

        divstart1 = item.find('<div class="s">') + len('<div class="s">')
        divstart2 = item.find('<div class="s hc">') + len('<div class="s hc">')

        #start = item.find('<div class="s">') + len('<div class="s">')
        if item.find('<div class="s">') > 0:
            start = divstart1
        elif item.find('<div class="s hc">') > 0:
            start = divstart2
        else:
            continue
        end = item.find('<br>')
        if end < 0:         
            continue   
        Summary = item[start:end]

        item = item[end + 4:]
        end = item.find('</cite>')
        if end < 0:
            continue    
        Url = item[:end]

        Title = Title.replace('<b>', '')
        Title = Title.replace('</b>', '')
        Title = Title.replace('<em>', '')
        Title = Title.replace('</em>', '')
        Summary = Summary.replace('<b>', '')
        Summary = Summary.replace('</b>', '')
        Summary = Summary.replace('<em>', '')
        Summary = Summary.replace('</em>', '')
        Url = Url.replace('<cite>', '')
        Url = Url.replace('</cite>', '')
        Url = Url.replace('<b>', '')
        Url = Url.replace('</b>', '')

        result = { 'Title': urllib.unquote(Title), 
                  'Summary': urllib.unquote(Summary), 
                  'Url': urllib.unquote(Url), 
                  'ClickUrl': urllib.unquote(ClickUrl)}        
        results.append(result)
    return results

def mmseg(text):
      a = ComplexAlgorithm(text)
      words = a.segment()
      ww = [unicode(w) for w in words]
      result = ' '.join(ww)
      #logging.debug(result)
      #self.response.out.write(result)
      return result


class UtcTzinfo(datetime.tzinfo):
    def utcoffset(self, dt):
        return datetime.timedelta(0)
    def dst(self, dt): 
        return datetime.timedelta(0)
    def tzname(self, dt): 
        return 'UTC'
    def olsen_name(self): 
        return 'UTC'

class ShTzinfo(datetime.tzinfo):
    def utcoffset(self, dt):
        return datetime.timedelta(hours=+8)
    def dst(self, dt):
        return datetime.timedelta(0)
    def tzname(self, dt):
        return 'GMT+08/Shanghai'
    def olsen_name(self):
        return 'Asia/Shanghai'

class PstTzinfo(datetime.tzinfo):
    def utcoffset(self, dt):
        return datetime.timedelta(hours=-8)
    def dst(self, dt):
        return datetime.timedelta(0)
    def tzname(self, dt):
        return 'PST+08PDT'
    def olsen_name(self):
        return 'US/Pacific'

TZINFOS = {
  'utc': UtcTzinfo(),
  'sh': ShTzinfo(),
  'pst': PstTzinfo(),
}

def translate(timestamp):
    """Translates a UTC datetime to the env_tz query parameter's time zone.
    Args:
        timestamp: A datetime instance.
    Returns:
      the datetime is the result.
    """
    translate_to = 'sh'
    utc = TZINFOS['utc']
    timestamp = timestamp.replace(tzinfo=utc)
    return timestamp.astimezone(TZINFOS[translate_to])

